package pira.parser.xml.citeseerx;

import java.io.CharArrayWriter;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.util.Map;

import org.neo4j.graphdb.GraphDatabaseService;
import org.neo4j.graphdb.Node;
import org.neo4j.graphdb.Transaction;
import org.neo4j.graphdb.index.Index;
import org.neo4j.graphdb.index.IndexHits;
import org.neo4j.graphdb.index.IndexManager;
import org.neo4j.kernel.EmbeddedGraphDatabase;
import org.xml.sax.Attributes;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
import org.xml.sax.XMLReader;
import org.xml.sax.helpers.DefaultHandler;
import org.xml.sax.helpers.XMLReaderFactory;

import pira.constants.PIRAConfig;
import pira.constants.PIRAConfig.MyRelationshipTypes;
import pira.db.ui.FullDBUI;
import pira.elements.Author;
import pira.elements.Paper;
import pira.utils.Log;

public class ParseCiteSeerXML extends DefaultHandler {

	public static long startTime;
	// constant for index
	public static final String PAPERS = "papers";
	public static final String AUTHORS = "authors";
	public static final String CITATIONS = "citations";

	// constant for elements in XML
	public static final String DOCUMENT = "document";
	public static final String TITLE = "title";
	public static final String AUTHOR = "author";
	public static final String NAME = "name";
	public static final String CLUSTERID = "clusterid";
	public static final String YEAR = "year";
	
	// node numbers
	private long paperNumber;
	private long authorNumber;
	
	// program log file
	private Log log;
	private Log error;

	// transaction for database
	private Transaction tx;

	// graph database service
	private GraphDatabaseService graphDb;

	// database index
	private IndexManager index;

	// index for papers
	public Index<Node> papers;

	// index for authors
	public Index<Node> authors;
	
	// file reader
	private FileReader fr;

	// current paper object reference
	private Paper currentPaper;
	private Node currentPNode;

	// current author object reference
	private Author currentAuthor;
	
	// flag to mark weather is reading paper info
	private static boolean thisPaper = false;
	
	// constructor
	public ParseCiteSeerXML() {
		log = new Log("Full_CiteSeerX_Log_v4.txt");
		error = new Log("Error.txt");
		Map<String, String> configuration = EmbeddedGraphDatabase
				.loadConfigurations(PIRAConfig.CONFIG);
		graphDb = new EmbeddedGraphDatabase(PIRAConfig.DBPATH, configuration);
		tx = graphDb.beginTx();
		try {
			index = graphDb.index();

			this.papers = index.forNodes(PAPERS);
			this.authors = index.forNodes(AUTHORS);
			
			createGlobalNode();
			createExternalNode();
			
			tx.success();
		} finally {
			tx.finish();
		}
	}

	// use to gather data from characters SAX
	private CharArrayWriter contents = new CharArrayWriter();

	// start to run
	public void run() {
		tx = graphDb.beginTx();
		paperNumber = 0;
		authorNumber = 0;
		
		File root = new File(PIRAConfig.XMLPATH);
		readDirectory(root, 0);
		
		globalRecord();
		
		shutDown();
	}
	
	// close program
	private void shutDown() {
		log.close();
		error.close();
		System.out.println("Programs end ...");
		tx.success();
		tx.finish();
		graphDb.shutdown();
	}
	
	// statistic record
	private void globalRecord() {
		// store global info in global node
		Node gNode = graphDb.getNodeById(PIRAConfig.GLOBALNODEID);
		gNode.setProperty(PIRAConfig.AUTHORNUMBER, authorNumber);
		gNode.setProperty(PIRAConfig.PAPERNUMBER, paperNumber);
		gNode.setProperty(PIRAConfig.ALLNODENUMBER, authorNumber + paperNumber
				+ 2);

		// record informations
		long elapsedTime = (System.currentTimeMillis() - startTime) / 1000;
		System.out.println(paperNumber);
		System.out.println("Time passed : " + elapsedTime + "s.");

		String str = paperNumber + "\t" + elapsedTime + "\t" + authorNumber
				+ "\n";
		log.write(str);
	}
	
	// create external node
	private long createExternalNode() {
		Node externalNode = graphDb.createNode();
		externalNode.setProperty(Paper.TITLE, PIRAConfig.EXTERNAL);

		long id = externalNode.getId();

		return id;
	}
	
	// create global node the metadata of the database
	private long createGlobalNode() {
		Node gnode = graphDb.createNode();
		gnode.setProperty(PIRAConfig.ALLNODENUMBER, 2);
		gnode.setProperty(PIRAConfig.PAPERNUMBER, 0);
		gnode.setProperty(PIRAConfig.AUTHORNUMBER, 0);
		return gnode.getId();
	}
	
	// override the DefaultHandler class to block the method of SAX accident
	// read the org.xml.sax.ContentHandler for more
	@Override
	public void startElement(String namespaceURI, String localName,
			String qName, Attributes attr) throws SAXException {
		contents.reset();

		// get the paper ID
		if (localName.equals(DOCUMENT)) {
			// create a new instance and add it to database
			currentPaper = new Paper();
			if (attr.getLength() > 0) {
				currentPaper.setPaperID(attr.getValue(PIRAConfig.ID));
			}
			
			addPaperToDB();
			thisPaper = true;
		}
		
		if (localName.equals(AUTHORS) || localName.equals(CITATIONS)) {
			thisPaper = false;
		}
		
		if (localName.equals(AUTHOR)) {
			// just create a new author instance
			currentAuthor = new Author();
			if (attr.getLength() > 0) {
				currentAuthor.setAuthorID(attr.getValue(PIRAConfig.ID));
			}
		}
	}

	@Override
	public void endElement(String namespaceURI, String localName, String qName)
			throws SAXException {
		if (localName.equals(CLUSTERID)) {
			if (thisPaper) {
				// set paper cluster id property
				currentPNode.setProperty(Paper.CLUSTERID, contents.toString());
				
				// update index of cluster id
				papers.add(currentPNode, Paper.CLUSTERID, contents.toString());
			}
		}
		
		if (localName.equals(TITLE)) {
			if (thisPaper) {
				currentPaper.setTitle(contents.toString());
				
				// add properties to the paper 
				currentPNode.setProperty(Paper.TITLE, currentPaper.getTitle());
			}
		}
		
		if (localName.equals(YEAR)) {
			if (thisPaper) {
				currentPNode.setProperty(Paper.YEAR, contents.toString());
				thisPaper = false;
			}
		}
		
		if (localName.equals(NAME)) {
			String name = contents.toString();
			currentAuthor.setFullName(name);
			
			// add current author to database
			addAuthorToDB();
//			currentPaper.getAuthors().add(currentAuthor);
		}
		
		if (localName.equals(AUTHORS)) {
			try {
				fr.close();
			} catch (IOException e) {
				e.printStackTrace();
			}
		}
	}

	@Override
	public void characters(char[] ch, int start, int length)
			throws SAXException {
		contents.write(ch, start, length);
	}

	// add current author and "wrote" relation to DB
	private Node addAuthorToDB() {
		Node node = null;
		// judge weather the author already exist in database
		// get the author id of the author
		String authorID = currentAuthor.getAuthorID();
		if (authorID.equals(PIRAConfig.NULL))
			authorID = "" + FullDBUI.normal(currentAuthor.getFullName()).hashCode();
		
		IndexHits<Node> hits = authors.get(Author.AUTHORID, authorID);
		// if found in database
		if (hits.size() > 0)
			node = hits.getSingle();
		else {
			// if not found , create new node
			node = graphDb.createNode();
			
			// author number increase 
			this.authorNumber++;
			
			// properties
			node.setProperty(Author.AUTHORID, authorID);
			node.setProperty(Author.FULLNAME, currentAuthor.getFullName());
			
			// use author id as index
			authors.add(node, Author.AUTHORID, authorID);
		}
		
		// "wrote" and "written_by" relationships
		node.createRelationshipTo(currentPNode, MyRelationshipTypes.WROTE);
		currentPNode.createRelationshipTo(node, MyRelationshipTypes.WRITTEN_BY);
		
		return node;
	}
	
	// add current paper to DB
	private Node addPaperToDB() {
		Node node = graphDb.createNode();
		
		// paper number increase
		this.paperNumber++;
//System.out.println(paperNumber);
		
		// set the current paper node as it
		this.currentPNode = node;
		
		// set paper id property
		node.setProperty(Paper.PAPERID, currentPaper.getPaperID());
		
		// use paper id as index
		papers.add(node, Paper.PAPERID, currentPaper.getPaperID());
		
		return node;
	}
	
	private void readFile(File xmlFile) {
		try {
			// create SAX 2 parser
			XMLReader xr = XMLReaderFactory.createXMLReader();

			// create file reader
			fr = new FileReader(xmlFile);
			
			// install ContentHandler．．．
			xr.setContentHandler(this);

			// parse file
			xr.parse(new InputSource(fr));

		} catch (IllegalStateException e1) {
			System.out.println("Index has been shut down.");
			System.out.println("Transformer from XML to DB end.");
		} catch (Exception e) {
//			System.out.println("RUN here");
			// record to log
			if (paperNumber % 10000 == 0) {
				
				globalRecord();
				
				// restart the transaction
				tx.success();
				tx.finish();
				tx = graphDb.beginTx();
				
				// end 
//				if (paperNumber == 20000) {
//					shutDown();
//					System.exit(1);
//				}
			}
		}
	}
	
	private void readDirectory(File directory, int depth) {
		File[] list = directory.listFiles();
		for (File f : list) {
			if (f.isDirectory()) {
				if (depth < 4)
					readDirectory(f, depth+1);
				else {
					File[] subversions = f.listFiles();
					if (subversions.length == 1) {
						readFile(subversions[0]);
					} else if (subversions.length == 2) {
						int l0 = subversions[0].getName().length();
						int l1 = subversions[1].getName().length();
						if (l0 > l1)
							readFile(subversions[0]);
						else
							readFile(subversions[1]);
					} else {
						File choice = subversions[0];
						int maxVersion = 0;
						for (File file : subversions) {
							int index = file.getName().indexOf('v');
							if (index != -1) {
								int version = -1;
								try {
									version = Integer.parseInt(file.getName().substring(index+1, index+2)); 
								} catch (NumberFormatException e) {
									error.write(file.getName() + "\n");
								}
								if (version > maxVersion) {
									maxVersion = version;
									choice = file;
								}
							} else
								continue;
						}
						readFile(choice);
					}
				}
					
			} else {
				readFile(f);
			}
		}
	}

	public static void main(String[] args) {
		startTime = System.currentTimeMillis();
		new ParseCiteSeerXML().run();
	}
}
